# coding: utf-8
import urllib
import urllib2
import cookielib
from HTMLParser import HTMLParser
import re
import xlrd
import xlwt
import string
from urllib2 import Request, urlopen, URLError, HTTPError
import requests

#======================================================
class myHtmlParser(HTMLParser):
    irstart=0
    irend=0
    result=[]
    data=[]
    isfinids=0
    iscleasspace=0
    def __init__(self):
        HTMLParser.__init__(self)
        self.flag=None
    def handle_starttag(self,tag,attrs):
        if self.irstart==1:
            if tag=='a':
                self.flag='a'
                for href,link in attrs:
                    if href=='href':
                        if self.isfinids==1:
                            if link!='/newsAction.do?method=viewNews&classId=020019980000000472&newsId=020010040000016998' and link!='/newsAction.do?method=viewNews&classId=020010350000000877&newsId=020010040000108745':
                                if link.find('/newsAction.do?method=viewNews&classId')>=0:
                                    self.result.append(link)
                        else:
                            self.result.append(link)





    def handle_data(self,data):
        if self.irend==1:
            sdata = data.strip()
            if sdata!='':
                if self.iscleasspace==1:
                    sdata=sdata.replace("\r\n","")
                    sdata=sdata.replace(" ","")
                    sdata=sdata.replace(' ','')
                self.data.append(sdata)
                #print 'data:'+sdata

#======================================================
#http://www.gdlr.gov.cn/newsAction.do?method=viewNews&classId=020010350000000680&newsId=020010040000205418
fdcinfolist=[]

findpage = 1
while findpage<=33:

    print str(findpage)+"获取所有id"
    cookie = cookielib.CookieJar()
    opener=urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie))
    url='http://www.gdlr.gov.cn/newsAction.do?method=searchNews'
    tilename = u'高明区'
    tilename = tilename.encode("gb2312")
    values={
        'classId':'null',
        'textGeneralType':'020010350000000680',
        'newsCatalogID':'020010350000000680',
        'isquery':'',
        'type':'',
        'type_name':'',
        'oldTitle':tilename,'oldTitle':'','titleName':tilename,'linkUrl':'',
        '020010350000000680':'','strTime':'','endTime':'','orderSign':'0','curPageNo':'1',
        'totalCnts':'322','totalPages':'33','cntPerPage':'10','SplitFlag':'1','orderBy':'null','descOrAsc':'desc','gotoPage':findpage}
    data = urllib.urlencode(values)
    req = urllib2.Request(url,data)
    req.add_header('Host','www.gdlr.gov.cn')
    req.add_header('Referer','http://www.gdlr.gov.cn/newsAction.do?method=searchNews&textGeneralType=020010350000000680')
    req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')
    response = opener.open(req)
    the_page=response.read()
    pstrl = unicode(the_page,"gb2312",'ignore').encode("utf8")
    response.close()
    m= myHtmlParser()
    m.irstart=1
    m.isfinids=1
    m.feed(pstrl)
    m.close()
    result = m.result
    for itemstr in result:
        fdcinfolist.append(itemstr)
    findpage+=1

print "删除重复前:"+str(len(fdcinfolist))
fdcinfolist =  list(set(fdcinfolist))
print "删除重复后:"+str(len(fdcinfolist))


#创建表格
f= xlwt.Workbook()#创建工作簿
sheetname = u"结果出让公告"
sheet = f.add_sheet(sheetname,cell_overwrite_ok=True)
#行头
print "创建表格"

readPos = -1
pppppp=1
for fdcid in fdcinfolist:
    readPos+=1
    print  "=获取id:"+fdcid
    url = 'http://www.gdlr.gov.cn'+fdcid
    print '浏览网页:'+url
    bskk =0
    botiaozhuan = 0
    try:
        headers = requests.head(url).headers

        for kkk in headers:
            if kkk=='location':
                url= headers['location']
                botiaozhuan=1
    except :
        botiaozhuan=0


    request = urllib2.Request(url)
    print '跳转网页:'+url
    if botiaozhuan==1:
        request.add_header('Host','www.landgd.com')
    else:
        request.add_header('Host',' www.gdlr.gov.cn')
    request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')

    try:
        response = urllib2.urlopen(request)
    except :
        continue

    print str(readPos)+"读取网址"+url
    vkstr = response.read()
    vkstr= unicode(vkstr,"gb2312",'ignore').encode("utf8")
    response.close()
    vstrs = vkstr.split('method="post" action="')
    wmguid = ''
    recorderguid=''
    if len(vstrs)>=2:
        vstrs = vstrs[1].split('" id="Form1">')
        if len(vstrs)>=2:
            newstr = vstrs[0].replace('workView.aspx?wmguid=','')
            isList = newstr.split('&amp;recorderguid=')
            wmguid=isList[0]
            recorderguid=isList[1]

         #http://www.landgd.com/DesktopModule/BizframeExtendMdl/workList/workView.aspx?wmguid=75c72564-ffd9-426a-954b-8ac2df0903b7&recorderguid=a8c91c21-9a30-4b8a-bcc4-8c528f364cb2
    url='http://www.landgd.com//ExtendModule/Platform/Module/ExternalServices/Form/FormView.aspx?wmguid='+wmguid +'&recorderguid='+recorderguid+'&bizUserName='

    if botiaozhuan==0:
        url = 'http://www.gdlr.gov.cn'+fdcid
    request = urllib2.Request(url)
    request.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.3; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0')
    print str(readPos)+"读取表格:"+url
    try:
        response = urllib2.urlopen(request)
    except :
        continue
    vkstr = response.read()

    vkstr= unicode(vkstr,"gb2312",'ignore').encode("utf8")

    response.close()
    m= myHtmlParser()
    m.data=[]
    m.irend=1
    m.feed(vkstr)
    m.close()
    nresult=[]
    nresult = m.data

    print "长度:"+str(len(nresult))

    ilks = 0
    print '============================================================================================='

    print '=======写入表格==========='


    #创建
    #
    istart=0
    iend = 0
    olps = 0
    while olps <len(nresult):
        fs = nresult[olps]
        if istart>0 and iend>0:
            break
        if fs =='供地结果信息':
            istart=olps
        if fs=='查看   供应结果[土地市场网]':
            iend=olps
        olps+=1

    print '开始位置:'+str(istart)
    print '结束位置:'+str(iend)

    rongjis=''
    rongjisx=''
    koindx = pppppp
    pppppp+=1
    print "第"+str(koindx)+'行'
    ilks = 0

    print '====================================================='
    for lllll in nresult:
        print str(koindx)+'输出:'+lllll
    print '====================================================='

    while ilks<len(nresult):
        finstr = nresult[ilks]
        if finstr=='行政区:':
            xzq= nresult[ilks+1]
            sheet.write(koindx,0,unicode(xzq,'utf-8'))
            print str(koindx)+finstr+":"+xzq
        if finstr=='项目名称:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,1,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='项目位置:' or finstr=='位置:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,2,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='面积(公倾):':
            xmm= nresult[ilks+1]
            sheet.write(koindx,3,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='供地方式:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,4,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='土地使用年限:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,5,unicode(xmm,'utf-8'))
        if finstr=='行业分类:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,6,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='土地使用权人:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,12,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='土地级别:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,7,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='成交价格(万元):':
            xmm= nresult[ilks+1]
            sheet.write(koindx,8,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='上限:':
            xmm= nresult[ilks+1]
            if xmm =='约定交地时间:' or xmm=='下限:' or xmm=='上限:':
                xmm='0'
            rongjisx='上限:'+xmm
            print str(koindx)+finstr+":"+xmm
        if finstr=='下限:':
            xmm= nresult[ilks+1]
            if xmm =='约定交地时间:' or xmm=='下限:'or xmm=='上限:':
                 xmm='0'
            print str(koindx)+finstr+":"+xmm
            rongjis='下限:'+xmm
        if finstr=='土地用途:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,11,unicode(xmm,'utf-8'))
            print str(koindx)+finstr+":"+xmm
        if finstr=='合同签订日期:' or finstr=='成交日期:' or finstr=='签订日期:':
            xmm= nresult[ilks+1]
            sheet.write(koindx,10,xmm)
            print str(koindx)+finstr+":"+xmm
        ilks+=1
    rstr = rongjis+' '+rongjisx

    sheet.write(koindx,9,unicode(rstr,'utf-8'))

sheet.write(0,0,u'行政区')
sheet.write(0,1,u'项目名称')
sheet.write(0,2,u'项目位置')
sheet.write(0,3,u'面积(公倾)')
sheet.write(0,4,u'供地方式')
sheet.write(0,5,u'土地使用年限')
sheet.write(0,6,u'行业分类')
sheet.write(0,7,u'土地级别')
sheet.write(0,8,u'成交价格(万元)')
sheet.write(0,9,u'容积')
sheet.write(0,10,u'合同签订日期')
sheet.write(0,11,u'土地用途')
sheet.write(0,12,u'土地使用权人:')


f.save( u'高明土地信息(结果出让公告).xls')